{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "vanillaDQN.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "wBfgbB2gJ-83",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import random\n",
        "import numpy as np\n",
        "from collections import deque\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.autograd as autograd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Buen0fyBKRIW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def mini_batch_train(env, agent, max_episodes, max_steps, batch_size):\n",
        "    episode_rewards = []\n",
        "\n",
        "    for episode in range(max_episodes):\n",
        "        state = env.reset()\n",
        "        episode_reward = 0\n",
        "\n",
        "        for step in range(max_steps):\n",
        "            action = agent.get_action(state)\n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "            agent.replay_buffer.push(state, action, reward, next_state, done)\n",
        "            episode_reward += reward\n",
        "\n",
        "            if len(agent.replay_buffer) > batch_size:\n",
        "                agent.update(batch_size)   \n",
        "\n",
        "            if done or step == max_steps-1:\n",
        "                episode_rewards.append(episode_reward)\n",
        "                print(\"Episode \" + str(episode) + \": \" + str(episode_reward))\n",
        "                break\n",
        "\n",
        "            state = next_state\n",
        "\n",
        "    return episode_rewards"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3ph-nYNMKNWx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BasicBuffer:\n",
        "\n",
        "    def __init__(self, max_size):\n",
        "        self.max_size = max_size\n",
        "        self.buffer = deque(maxlen=max_size)\n",
        "\n",
        "    def push(self, state, action, reward, next_state, done):\n",
        "        experience = (state, action, np.array([reward]), next_state, done)\n",
        "        self.buffer.append(experience)\n",
        "\n",
        "    def sample(self, batch_size):\n",
        "        state_batch = []\n",
        "        action_batch = []\n",
        "        reward_batch = []\n",
        "        next_state_batch = []\n",
        "        done_batch = []\n",
        "\n",
        "        batch = random.sample(self.buffer, batch_size)\n",
        "\n",
        "        for experience in batch:\n",
        "            state, action, reward, next_state, done = experience\n",
        "            state_batch.append(state)\n",
        "            action_batch.append(action)\n",
        "            reward_batch.append(reward)\n",
        "            next_state_batch.append(next_state)\n",
        "            done_batch.append(done)\n",
        "\n",
        "        return (state_batch, action_batch, reward_batch, next_state_batch, done_batch)\n",
        "\n",
        "    def sample_sequence(self, batch_size):\n",
        "        state_batch = []\n",
        "        action_batch = []\n",
        "        reward_batch = []\n",
        "        next_state_batch = []\n",
        "        done_batch = []\n",
        "\n",
        "        min_start = len(self.buffer) - batch_size\n",
        "        start = np.random.randint(0, min_start)\n",
        "\n",
        "        for sample in range(start, start + batch_size):\n",
        "            state, action, reward, next_state, done = self.buffer[start]\n",
        "            state, action, reward, next_state, done = experience\n",
        "            state_batch.append(state)\n",
        "            action_batch.append(action)\n",
        "            reward_batch.append(reward)\n",
        "            next_state_batch.append(next_state)\n",
        "            done_batch.append(done)\n",
        "\n",
        "        return (state_batch, action_batch, reward_batch, next_state_batch, done_batch)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.buffer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lD3J5jnpKT9a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ConvDQN(nn.Module):\n",
        "    \n",
        "    def __init__(self, input_dim, output_dim):\n",
        "        super(ConvDQN, self).__init__()\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        self.fc_input_dim = self.feature_size()\n",
        "        \n",
        "        self.conv = nn.Sequential(\n",
        "            nn.Conv2d(self.input_dim[0], 32, kernel_size=8, stride=4),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(32, 64, kernel_size=4, stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(64, 64, kernel_size=3, stride=1),\n",
        "            nn.ReLU()\n",
        "        )\n",
        "\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(self.fc_input_dim, 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(128, 256),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(256, self.output_dim)\n",
        "        )\n",
        "\n",
        "    def forward(self, state):\n",
        "        features = self.conv_net(state)\n",
        "        features = features.view(features.size(0), -1)\n",
        "        qvals = self.fc(features)\n",
        "        return qvals\n",
        "\n",
        "    def feature_size(self):\n",
        "        return self.conv_net(autograd.Variable(torch.zeros(1, *self.input_dim))).view(1, -1).size(1)\n",
        "\n",
        "\n",
        "class DQN(nn.Module):\n",
        "    \n",
        "    def __init__(self, input_dim, output_dim):\n",
        "        super(DQN, self).__init__()\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(self.input_dim[0], 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(128, 256),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(256, self.output_dim)\n",
        "        )\n",
        "\n",
        "    def forward(self, state):\n",
        "        qvals = self.fc(state)\n",
        "        return qvals"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ISFi-KsNKbiX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class DQNAgent:\n",
        "\n",
        "    def __init__(self, env, use_conv=True, learning_rate=3e-4, gamma=0.99, buffer_size=10000):\n",
        "        self.env = env\n",
        "        self.learning_rate = learning_rate\n",
        "        self.gamma = gamma\n",
        "        self.replay_buffer = BasicBuffer(max_size=buffer_size)\n",
        "        self.device = \"cpu\"\n",
        "        if torch.cuda.is_available():\n",
        "            self.device = \"cuda\"\n",
        "\n",
        "        self.use_conv = use_conv\n",
        "        if self.use_conv:\n",
        "            self.model = ConvDQN(env.observation_space.shape, env.action_space.n).to(self.device)\n",
        "        else:\n",
        "            self.model = DQN(env.observation_space.shape, env.action_space.n).to(self.device)\n",
        "\n",
        "        self.optimizer = torch.optim.Adam(self.model.parameters())\n",
        "        self.MSE_loss = nn.MSELoss()\n",
        "\n",
        "    def get_action(self, state, eps=0.20):\n",
        "        state = torch.FloatTensor(state).float().unsqueeze(0).to(self.device)\n",
        "        qvals = self.model.forward(state)\n",
        "        action = np.argmax(qvals.cpu().detach().numpy())\n",
        "        \n",
        "        if(np.random.randn() < eps):\n",
        "            return self.env.action_space.sample()\n",
        "\n",
        "        return action\n",
        "\n",
        "    def compute_loss(self, batch):\n",
        "        states, actions, rewards, next_states, dones = batch\n",
        "        states = torch.FloatTensor(states).to(self.device)\n",
        "        actions = torch.LongTensor(actions).to(self.device)\n",
        "        rewards = torch.FloatTensor(rewards).to(self.device)\n",
        "        next_states = torch.FloatTensor(next_states).to(self.device)\n",
        "        dones = torch.FloatTensor(dones)\n",
        "\n",
        "        curr_Q = self.model.forward(states).gather(1, actions.unsqueeze(1))\n",
        "        curr_Q = curr_Q.squeeze(1)\n",
        "        next_Q = self.model.forward(next_states)\n",
        "        max_next_Q = torch.max(next_Q, 1)[0]\n",
        "        expected_Q = rewards.squeeze(1) + self.gamma * max_next_Q\n",
        "\n",
        "        loss = self.MSE_loss(curr_Q, expected_Q)\n",
        "        return loss\n",
        "\n",
        "    def update(self, batch_size):\n",
        "        batch = self.replay_buffer.sample(batch_size)\n",
        "        loss = self.compute_loss(batch)\n",
        "\n",
        "        self.optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        self.optimizer.step()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGxmFHF0KiDm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "cfdc6ef5-950a-4ce4-89ae-175a2050ff62"
      },
      "source": [
        "import gym\n",
        "\n",
        "env_id = \"CartPole-v0\"\n",
        "MAX_EPISODES = 1000\n",
        "MAX_STEPS = 500\n",
        "BATCH_SIZE = 32\n",
        "\n",
        "env = gym.make(env_id)\n",
        "agent = DQNAgent(env, use_conv=False)\n",
        "episode_rewards = mini_batch_train(env, agent, MAX_EPISODES, MAX_STEPS, BATCH_SIZE)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Episode 0: 11.0\n",
            "Episode 1: 9.0\n",
            "Episode 2: 20.0\n",
            "Episode 3: 17.0\n",
            "Episode 4: 17.0\n",
            "Episode 5: 12.0\n",
            "Episode 6: 16.0\n",
            "Episode 7: 42.0\n",
            "Episode 8: 20.0\n",
            "Episode 9: 18.0\n",
            "Episode 10: 32.0\n",
            "Episode 11: 10.0\n",
            "Episode 12: 48.0\n",
            "Episode 13: 51.0\n",
            "Episode 14: 21.0\n",
            "Episode 15: 27.0\n",
            "Episode 16: 13.0\n",
            "Episode 17: 16.0\n",
            "Episode 18: 21.0\n",
            "Episode 19: 29.0\n",
            "Episode 20: 18.0\n",
            "Episode 21: 12.0\n",
            "Episode 22: 22.0\n",
            "Episode 23: 13.0\n",
            "Episode 24: 25.0\n",
            "Episode 25: 13.0\n",
            "Episode 26: 46.0\n",
            "Episode 27: 21.0\n",
            "Episode 28: 38.0\n",
            "Episode 29: 15.0\n",
            "Episode 30: 18.0\n",
            "Episode 31: 77.0\n",
            "Episode 32: 37.0\n",
            "Episode 33: 44.0\n",
            "Episode 34: 21.0\n",
            "Episode 35: 29.0\n",
            "Episode 36: 20.0\n",
            "Episode 37: 60.0\n",
            "Episode 38: 77.0\n",
            "Episode 39: 37.0\n",
            "Episode 40: 47.0\n",
            "Episode 41: 44.0\n",
            "Episode 42: 17.0\n",
            "Episode 43: 19.0\n",
            "Episode 44: 17.0\n",
            "Episode 45: 17.0\n",
            "Episode 46: 20.0\n",
            "Episode 47: 65.0\n",
            "Episode 48: 31.0\n",
            "Episode 49: 35.0\n",
            "Episode 50: 23.0\n",
            "Episode 51: 20.0\n",
            "Episode 52: 41.0\n",
            "Episode 53: 18.0\n",
            "Episode 54: 78.0\n",
            "Episode 55: 47.0\n",
            "Episode 56: 17.0\n",
            "Episode 57: 16.0\n",
            "Episode 58: 64.0\n",
            "Episode 59: 93.0\n",
            "Episode 60: 88.0\n",
            "Episode 61: 84.0\n",
            "Episode 62: 107.0\n",
            "Episode 63: 139.0\n",
            "Episode 64: 68.0\n",
            "Episode 65: 109.0\n",
            "Episode 66: 21.0\n",
            "Episode 67: 27.0\n",
            "Episode 68: 106.0\n",
            "Episode 69: 38.0\n",
            "Episode 70: 71.0\n",
            "Episode 71: 52.0\n",
            "Episode 72: 37.0\n",
            "Episode 73: 187.0\n",
            "Episode 74: 25.0\n",
            "Episode 75: 21.0\n",
            "Episode 76: 60.0\n",
            "Episode 77: 24.0\n",
            "Episode 78: 52.0\n",
            "Episode 79: 92.0\n",
            "Episode 80: 13.0\n",
            "Episode 81: 90.0\n",
            "Episode 82: 90.0\n",
            "Episode 83: 136.0\n",
            "Episode 84: 29.0\n",
            "Episode 85: 93.0\n",
            "Episode 86: 48.0\n",
            "Episode 87: 56.0\n",
            "Episode 88: 39.0\n",
            "Episode 89: 49.0\n",
            "Episode 90: 100.0\n",
            "Episode 91: 100.0\n",
            "Episode 92: 91.0\n",
            "Episode 93: 34.0\n",
            "Episode 94: 95.0\n",
            "Episode 95: 77.0\n",
            "Episode 96: 22.0\n",
            "Episode 97: 63.0\n",
            "Episode 98: 65.0\n",
            "Episode 99: 18.0\n",
            "Episode 100: 12.0\n",
            "Episode 101: 23.0\n",
            "Episode 102: 13.0\n",
            "Episode 103: 38.0\n",
            "Episode 104: 14.0\n",
            "Episode 105: 53.0\n",
            "Episode 106: 42.0\n",
            "Episode 107: 73.0\n",
            "Episode 108: 98.0\n",
            "Episode 109: 115.0\n",
            "Episode 110: 10.0\n",
            "Episode 111: 33.0\n",
            "Episode 112: 35.0\n",
            "Episode 113: 30.0\n",
            "Episode 114: 29.0\n",
            "Episode 115: 42.0\n",
            "Episode 116: 62.0\n",
            "Episode 117: 34.0\n",
            "Episode 118: 30.0\n",
            "Episode 119: 78.0\n",
            "Episode 120: 17.0\n",
            "Episode 121: 32.0\n",
            "Episode 122: 38.0\n",
            "Episode 123: 12.0\n",
            "Episode 124: 18.0\n",
            "Episode 125: 20.0\n",
            "Episode 126: 20.0\n",
            "Episode 127: 53.0\n",
            "Episode 128: 11.0\n",
            "Episode 129: 19.0\n",
            "Episode 130: 19.0\n",
            "Episode 131: 18.0\n",
            "Episode 132: 26.0\n",
            "Episode 133: 11.0\n",
            "Episode 134: 32.0\n",
            "Episode 135: 32.0\n",
            "Episode 136: 31.0\n",
            "Episode 137: 44.0\n",
            "Episode 138: 32.0\n",
            "Episode 139: 12.0\n",
            "Episode 140: 26.0\n",
            "Episode 141: 76.0\n",
            "Episode 142: 30.0\n",
            "Episode 143: 58.0\n",
            "Episode 144: 98.0\n",
            "Episode 145: 66.0\n",
            "Episode 146: 59.0\n",
            "Episode 147: 107.0\n",
            "Episode 148: 55.0\n",
            "Episode 149: 53.0\n",
            "Episode 150: 48.0\n",
            "Episode 151: 87.0\n",
            "Episode 152: 52.0\n",
            "Episode 153: 54.0\n",
            "Episode 154: 17.0\n",
            "Episode 155: 81.0\n",
            "Episode 156: 40.0\n",
            "Episode 157: 131.0\n",
            "Episode 158: 50.0\n",
            "Episode 159: 109.0\n",
            "Episode 160: 52.0\n",
            "Episode 161: 20.0\n",
            "Episode 162: 28.0\n",
            "Episode 163: 92.0\n",
            "Episode 164: 51.0\n",
            "Episode 165: 115.0\n",
            "Episode 166: 65.0\n",
            "Episode 167: 90.0\n",
            "Episode 168: 32.0\n",
            "Episode 169: 16.0\n",
            "Episode 170: 73.0\n",
            "Episode 171: 61.0\n",
            "Episode 172: 30.0\n",
            "Episode 173: 40.0\n",
            "Episode 174: 28.0\n",
            "Episode 175: 16.0\n",
            "Episode 176: 17.0\n",
            "Episode 177: 28.0\n",
            "Episode 178: 36.0\n",
            "Episode 179: 33.0\n",
            "Episode 180: 27.0\n",
            "Episode 181: 11.0\n",
            "Episode 182: 87.0\n",
            "Episode 183: 57.0\n",
            "Episode 184: 13.0\n",
            "Episode 185: 31.0\n",
            "Episode 186: 18.0\n",
            "Episode 187: 10.0\n",
            "Episode 188: 17.0\n",
            "Episode 189: 13.0\n",
            "Episode 190: 25.0\n",
            "Episode 191: 13.0\n",
            "Episode 192: 19.0\n",
            "Episode 193: 9.0\n",
            "Episode 194: 11.0\n",
            "Episode 195: 11.0\n",
            "Episode 196: 11.0\n",
            "Episode 197: 20.0\n",
            "Episode 198: 14.0\n",
            "Episode 199: 10.0\n",
            "Episode 200: 18.0\n",
            "Episode 201: 36.0\n",
            "Episode 202: 16.0\n",
            "Episode 203: 14.0\n",
            "Episode 204: 9.0\n",
            "Episode 205: 14.0\n",
            "Episode 206: 23.0\n",
            "Episode 207: 26.0\n",
            "Episode 208: 22.0\n",
            "Episode 209: 9.0\n",
            "Episode 210: 11.0\n",
            "Episode 211: 14.0\n",
            "Episode 212: 18.0\n",
            "Episode 213: 26.0\n",
            "Episode 214: 11.0\n",
            "Episode 215: 9.0\n",
            "Episode 216: 24.0\n",
            "Episode 217: 19.0\n",
            "Episode 218: 34.0\n",
            "Episode 219: 16.0\n",
            "Episode 220: 12.0\n",
            "Episode 221: 10.0\n",
            "Episode 222: 13.0\n",
            "Episode 223: 11.0\n",
            "Episode 224: 17.0\n",
            "Episode 225: 15.0\n",
            "Episode 226: 9.0\n",
            "Episode 227: 37.0\n",
            "Episode 228: 49.0\n",
            "Episode 229: 19.0\n",
            "Episode 230: 13.0\n",
            "Episode 231: 20.0\n",
            "Episode 232: 15.0\n",
            "Episode 233: 33.0\n",
            "Episode 234: 14.0\n",
            "Episode 235: 17.0\n",
            "Episode 236: 24.0\n",
            "Episode 237: 17.0\n",
            "Episode 238: 14.0\n",
            "Episode 239: 12.0\n",
            "Episode 240: 17.0\n",
            "Episode 241: 16.0\n",
            "Episode 242: 9.0\n",
            "Episode 243: 9.0\n",
            "Episode 244: 10.0\n",
            "Episode 245: 12.0\n",
            "Episode 246: 9.0\n",
            "Episode 247: 12.0\n",
            "Episode 248: 11.0\n",
            "Episode 249: 10.0\n",
            "Episode 250: 10.0\n",
            "Episode 251: 43.0\n",
            "Episode 252: 14.0\n",
            "Episode 253: 14.0\n",
            "Episode 254: 11.0\n",
            "Episode 255: 17.0\n",
            "Episode 256: 16.0\n",
            "Episode 257: 13.0\n",
            "Episode 258: 60.0\n",
            "Episode 259: 26.0\n",
            "Episode 260: 13.0\n",
            "Episode 261: 12.0\n",
            "Episode 262: 9.0\n",
            "Episode 263: 8.0\n",
            "Episode 264: 12.0\n",
            "Episode 265: 19.0\n",
            "Episode 266: 20.0\n",
            "Episode 267: 10.0\n",
            "Episode 268: 14.0\n",
            "Episode 269: 10.0\n",
            "Episode 270: 11.0\n",
            "Episode 271: 10.0\n",
            "Episode 272: 17.0\n",
            "Episode 273: 11.0\n",
            "Episode 274: 20.0\n",
            "Episode 275: 23.0\n",
            "Episode 276: 15.0\n",
            "Episode 277: 14.0\n",
            "Episode 278: 13.0\n",
            "Episode 279: 17.0\n",
            "Episode 280: 11.0\n",
            "Episode 281: 11.0\n",
            "Episode 282: 13.0\n",
            "Episode 283: 16.0\n",
            "Episode 284: 11.0\n",
            "Episode 285: 14.0\n",
            "Episode 286: 11.0\n",
            "Episode 287: 11.0\n",
            "Episode 288: 11.0\n",
            "Episode 289: 11.0\n",
            "Episode 290: 21.0\n",
            "Episode 291: 12.0\n",
            "Episode 292: 11.0\n",
            "Episode 293: 15.0\n",
            "Episode 294: 11.0\n",
            "Episode 295: 65.0\n",
            "Episode 296: 19.0\n",
            "Episode 297: 13.0\n",
            "Episode 298: 11.0\n",
            "Episode 299: 23.0\n",
            "Episode 300: 10.0\n",
            "Episode 301: 21.0\n",
            "Episode 302: 11.0\n",
            "Episode 303: 22.0\n",
            "Episode 304: 10.0\n",
            "Episode 305: 32.0\n",
            "Episode 306: 12.0\n",
            "Episode 307: 30.0\n",
            "Episode 308: 15.0\n",
            "Episode 309: 10.0\n",
            "Episode 310: 11.0\n",
            "Episode 311: 22.0\n",
            "Episode 312: 11.0\n",
            "Episode 313: 15.0\n",
            "Episode 314: 26.0\n",
            "Episode 315: 11.0\n",
            "Episode 316: 38.0\n",
            "Episode 317: 13.0\n",
            "Episode 318: 11.0\n",
            "Episode 319: 11.0\n",
            "Episode 320: 14.0\n",
            "Episode 321: 16.0\n",
            "Episode 322: 14.0\n",
            "Episode 323: 10.0\n",
            "Episode 324: 10.0\n",
            "Episode 325: 9.0\n",
            "Episode 326: 19.0\n",
            "Episode 327: 26.0\n",
            "Episode 328: 14.0\n",
            "Episode 329: 13.0\n",
            "Episode 330: 14.0\n",
            "Episode 331: 13.0\n",
            "Episode 332: 10.0\n",
            "Episode 333: 20.0\n",
            "Episode 334: 15.0\n",
            "Episode 335: 13.0\n",
            "Episode 336: 45.0\n",
            "Episode 337: 20.0\n",
            "Episode 338: 9.0\n",
            "Episode 339: 11.0\n",
            "Episode 340: 11.0\n",
            "Episode 341: 10.0\n",
            "Episode 342: 33.0\n",
            "Episode 343: 11.0\n",
            "Episode 344: 13.0\n",
            "Episode 345: 33.0\n",
            "Episode 346: 11.0\n",
            "Episode 347: 13.0\n",
            "Episode 348: 9.0\n",
            "Episode 349: 38.0\n",
            "Episode 350: 12.0\n",
            "Episode 351: 10.0\n",
            "Episode 352: 11.0\n",
            "Episode 353: 14.0\n",
            "Episode 354: 13.0\n",
            "Episode 355: 19.0\n",
            "Episode 356: 9.0\n",
            "Episode 357: 15.0\n",
            "Episode 358: 36.0\n",
            "Episode 359: 12.0\n",
            "Episode 360: 23.0\n",
            "Episode 361: 10.0\n",
            "Episode 362: 54.0\n",
            "Episode 363: 19.0\n",
            "Episode 364: 14.0\n",
            "Episode 365: 28.0\n",
            "Episode 366: 28.0\n",
            "Episode 367: 10.0\n",
            "Episode 368: 21.0\n",
            "Episode 369: 15.0\n",
            "Episode 370: 46.0\n",
            "Episode 371: 13.0\n",
            "Episode 372: 15.0\n",
            "Episode 373: 10.0\n",
            "Episode 374: 13.0\n",
            "Episode 375: 10.0\n",
            "Episode 376: 12.0\n",
            "Episode 377: 13.0\n",
            "Episode 378: 12.0\n",
            "Episode 379: 13.0\n",
            "Episode 380: 16.0\n",
            "Episode 381: 17.0\n",
            "Episode 382: 46.0\n",
            "Episode 383: 11.0\n",
            "Episode 384: 46.0\n",
            "Episode 385: 20.0\n",
            "Episode 386: 25.0\n",
            "Episode 387: 9.0\n",
            "Episode 388: 15.0\n",
            "Episode 389: 32.0\n",
            "Episode 390: 13.0\n",
            "Episode 391: 15.0\n",
            "Episode 392: 10.0\n",
            "Episode 393: 50.0\n",
            "Episode 394: 16.0\n",
            "Episode 395: 10.0\n",
            "Episode 396: 19.0\n",
            "Episode 397: 20.0\n",
            "Episode 398: 12.0\n",
            "Episode 399: 11.0\n",
            "Episode 400: 16.0\n",
            "Episode 401: 11.0\n",
            "Episode 402: 21.0\n",
            "Episode 403: 28.0\n",
            "Episode 404: 10.0\n",
            "Episode 405: 10.0\n",
            "Episode 406: 10.0\n",
            "Episode 407: 10.0\n",
            "Episode 408: 16.0\n",
            "Episode 409: 15.0\n",
            "Episode 410: 11.0\n",
            "Episode 411: 45.0\n",
            "Episode 412: 35.0\n",
            "Episode 413: 10.0\n",
            "Episode 414: 20.0\n",
            "Episode 415: 9.0\n",
            "Episode 416: 29.0\n",
            "Episode 417: 40.0\n",
            "Episode 418: 16.0\n",
            "Episode 419: 19.0\n",
            "Episode 420: 11.0\n",
            "Episode 421: 51.0\n",
            "Episode 422: 14.0\n",
            "Episode 423: 11.0\n",
            "Episode 424: 15.0\n",
            "Episode 425: 10.0\n",
            "Episode 426: 10.0\n",
            "Episode 427: 11.0\n",
            "Episode 428: 11.0\n",
            "Episode 429: 9.0\n",
            "Episode 430: 10.0\n",
            "Episode 431: 21.0\n",
            "Episode 432: 9.0\n",
            "Episode 433: 10.0\n",
            "Episode 434: 10.0\n",
            "Episode 435: 25.0\n",
            "Episode 436: 10.0\n",
            "Episode 437: 12.0\n",
            "Episode 438: 14.0\n",
            "Episode 439: 17.0\n",
            "Episode 440: 9.0\n",
            "Episode 441: 9.0\n",
            "Episode 442: 19.0\n",
            "Episode 443: 20.0\n",
            "Episode 444: 13.0\n",
            "Episode 445: 9.0\n",
            "Episode 446: 11.0\n",
            "Episode 447: 14.0\n",
            "Episode 448: 9.0\n",
            "Episode 449: 13.0\n",
            "Episode 450: 29.0\n",
            "Episode 451: 11.0\n",
            "Episode 452: 14.0\n",
            "Episode 453: 14.0\n",
            "Episode 454: 12.0\n",
            "Episode 455: 12.0\n",
            "Episode 456: 12.0\n",
            "Episode 457: 16.0\n",
            "Episode 458: 16.0\n",
            "Episode 459: 14.0\n",
            "Episode 460: 12.0\n",
            "Episode 461: 12.0\n",
            "Episode 462: 9.0\n",
            "Episode 463: 12.0\n",
            "Episode 464: 25.0\n",
            "Episode 465: 11.0\n",
            "Episode 466: 26.0\n",
            "Episode 467: 23.0\n",
            "Episode 468: 24.0\n",
            "Episode 469: 24.0\n",
            "Episode 470: 10.0\n",
            "Episode 471: 10.0\n",
            "Episode 472: 51.0\n",
            "Episode 473: 15.0\n",
            "Episode 474: 9.0\n",
            "Episode 475: 21.0\n",
            "Episode 476: 14.0\n",
            "Episode 477: 10.0\n",
            "Episode 478: 15.0\n",
            "Episode 479: 15.0\n",
            "Episode 480: 21.0\n",
            "Episode 481: 23.0\n",
            "Episode 482: 17.0\n",
            "Episode 483: 16.0\n",
            "Episode 484: 16.0\n",
            "Episode 485: 19.0\n",
            "Episode 486: 11.0\n",
            "Episode 487: 12.0\n",
            "Episode 488: 12.0\n",
            "Episode 489: 28.0\n",
            "Episode 490: 13.0\n",
            "Episode 491: 9.0\n",
            "Episode 492: 10.0\n",
            "Episode 493: 16.0\n",
            "Episode 494: 12.0\n",
            "Episode 495: 12.0\n",
            "Episode 496: 16.0\n",
            "Episode 497: 9.0\n",
            "Episode 498: 12.0\n",
            "Episode 499: 42.0\n",
            "Episode 500: 22.0\n",
            "Episode 501: 27.0\n",
            "Episode 502: 12.0\n",
            "Episode 503: 18.0\n",
            "Episode 504: 15.0\n",
            "Episode 505: 20.0\n",
            "Episode 506: 12.0\n",
            "Episode 507: 20.0\n",
            "Episode 508: 24.0\n",
            "Episode 509: 18.0\n",
            "Episode 510: 15.0\n",
            "Episode 511: 14.0\n",
            "Episode 512: 17.0\n",
            "Episode 513: 23.0\n",
            "Episode 514: 20.0\n",
            "Episode 515: 14.0\n",
            "Episode 516: 34.0\n",
            "Episode 517: 19.0\n",
            "Episode 518: 13.0\n",
            "Episode 519: 34.0\n",
            "Episode 520: 22.0\n",
            "Episode 521: 36.0\n",
            "Episode 522: 16.0\n",
            "Episode 523: 11.0\n",
            "Episode 524: 11.0\n",
            "Episode 525: 11.0\n",
            "Episode 526: 12.0\n",
            "Episode 527: 10.0\n",
            "Episode 528: 9.0\n",
            "Episode 529: 34.0\n",
            "Episode 530: 15.0\n",
            "Episode 531: 8.0\n",
            "Episode 532: 12.0\n",
            "Episode 533: 8.0\n",
            "Episode 534: 12.0\n",
            "Episode 535: 14.0\n",
            "Episode 536: 12.0\n",
            "Episode 537: 11.0\n",
            "Episode 538: 13.0\n",
            "Episode 539: 13.0\n",
            "Episode 540: 18.0\n",
            "Episode 541: 25.0\n",
            "Episode 542: 29.0\n",
            "Episode 543: 16.0\n",
            "Episode 544: 10.0\n",
            "Episode 545: 10.0\n",
            "Episode 546: 18.0\n",
            "Episode 547: 11.0\n",
            "Episode 548: 20.0\n",
            "Episode 549: 16.0\n",
            "Episode 550: 10.0\n",
            "Episode 551: 16.0\n",
            "Episode 552: 18.0\n",
            "Episode 553: 13.0\n",
            "Episode 554: 19.0\n",
            "Episode 555: 11.0\n",
            "Episode 556: 9.0\n",
            "Episode 557: 15.0\n",
            "Episode 558: 15.0\n",
            "Episode 559: 10.0\n",
            "Episode 560: 13.0\n",
            "Episode 561: 41.0\n",
            "Episode 562: 26.0\n",
            "Episode 563: 18.0\n",
            "Episode 564: 16.0\n",
            "Episode 565: 13.0\n",
            "Episode 566: 24.0\n",
            "Episode 567: 10.0\n",
            "Episode 568: 13.0\n",
            "Episode 569: 24.0\n",
            "Episode 570: 13.0\n",
            "Episode 571: 14.0\n",
            "Episode 572: 18.0\n",
            "Episode 573: 17.0\n",
            "Episode 574: 14.0\n",
            "Episode 575: 21.0\n",
            "Episode 576: 11.0\n",
            "Episode 577: 28.0\n",
            "Episode 578: 16.0\n",
            "Episode 579: 17.0\n",
            "Episode 580: 10.0\n",
            "Episode 581: 10.0\n",
            "Episode 582: 11.0\n",
            "Episode 583: 14.0\n",
            "Episode 584: 10.0\n",
            "Episode 585: 17.0\n",
            "Episode 586: 14.0\n",
            "Episode 587: 23.0\n",
            "Episode 588: 19.0\n",
            "Episode 589: 16.0\n",
            "Episode 590: 14.0\n",
            "Episode 591: 21.0\n",
            "Episode 592: 12.0\n",
            "Episode 593: 14.0\n",
            "Episode 594: 12.0\n",
            "Episode 595: 11.0\n",
            "Episode 596: 14.0\n",
            "Episode 597: 10.0\n",
            "Episode 598: 15.0\n",
            "Episode 599: 16.0\n",
            "Episode 600: 36.0\n",
            "Episode 601: 17.0\n",
            "Episode 602: 11.0\n",
            "Episode 603: 12.0\n",
            "Episode 604: 11.0\n",
            "Episode 605: 9.0\n",
            "Episode 606: 22.0\n",
            "Episode 607: 13.0\n",
            "Episode 608: 13.0\n",
            "Episode 609: 12.0\n",
            "Episode 610: 16.0\n",
            "Episode 611: 9.0\n",
            "Episode 612: 18.0\n",
            "Episode 613: 13.0\n",
            "Episode 614: 9.0\n",
            "Episode 615: 16.0\n",
            "Episode 616: 14.0\n",
            "Episode 617: 11.0\n",
            "Episode 618: 27.0\n",
            "Episode 619: 9.0\n",
            "Episode 620: 14.0\n",
            "Episode 621: 22.0\n",
            "Episode 622: 21.0\n",
            "Episode 623: 11.0\n",
            "Episode 624: 10.0\n",
            "Episode 625: 28.0\n",
            "Episode 626: 10.0\n",
            "Episode 627: 19.0\n",
            "Episode 628: 11.0\n",
            "Episode 629: 11.0\n",
            "Episode 630: 14.0\n",
            "Episode 631: 11.0\n",
            "Episode 632: 14.0\n",
            "Episode 633: 18.0\n",
            "Episode 634: 17.0\n",
            "Episode 635: 17.0\n",
            "Episode 636: 32.0\n",
            "Episode 637: 20.0\n",
            "Episode 638: 19.0\n",
            "Episode 639: 15.0\n",
            "Episode 640: 10.0\n",
            "Episode 641: 27.0\n",
            "Episode 642: 15.0\n",
            "Episode 643: 13.0\n",
            "Episode 644: 12.0\n",
            "Episode 645: 11.0\n",
            "Episode 646: 12.0\n",
            "Episode 647: 18.0\n",
            "Episode 648: 14.0\n",
            "Episode 649: 13.0\n",
            "Episode 650: 12.0\n",
            "Episode 651: 16.0\n",
            "Episode 652: 12.0\n",
            "Episode 653: 18.0\n",
            "Episode 654: 25.0\n",
            "Episode 655: 24.0\n",
            "Episode 656: 22.0\n",
            "Episode 657: 26.0\n",
            "Episode 658: 18.0\n",
            "Episode 659: 15.0\n",
            "Episode 660: 20.0\n",
            "Episode 661: 15.0\n",
            "Episode 662: 15.0\n",
            "Episode 663: 11.0\n",
            "Episode 664: 17.0\n",
            "Episode 665: 16.0\n",
            "Episode 666: 13.0\n",
            "Episode 667: 86.0\n",
            "Episode 668: 12.0\n",
            "Episode 669: 25.0\n",
            "Episode 670: 9.0\n",
            "Episode 671: 18.0\n",
            "Episode 672: 15.0\n",
            "Episode 673: 13.0\n",
            "Episode 674: 10.0\n",
            "Episode 675: 23.0\n",
            "Episode 676: 16.0\n",
            "Episode 677: 30.0\n",
            "Episode 678: 10.0\n",
            "Episode 679: 11.0\n",
            "Episode 680: 16.0\n",
            "Episode 681: 14.0\n",
            "Episode 682: 16.0\n",
            "Episode 683: 10.0\n",
            "Episode 684: 42.0\n",
            "Episode 685: 16.0\n",
            "Episode 686: 11.0\n",
            "Episode 687: 23.0\n",
            "Episode 688: 10.0\n",
            "Episode 689: 8.0\n",
            "Episode 690: 21.0\n",
            "Episode 691: 11.0\n",
            "Episode 692: 15.0\n",
            "Episode 693: 17.0\n",
            "Episode 694: 15.0\n",
            "Episode 695: 10.0\n",
            "Episode 696: 10.0\n",
            "Episode 697: 10.0\n",
            "Episode 698: 9.0\n",
            "Episode 699: 36.0\n",
            "Episode 700: 13.0\n",
            "Episode 701: 12.0\n",
            "Episode 702: 10.0\n",
            "Episode 703: 20.0\n",
            "Episode 704: 16.0\n",
            "Episode 705: 13.0\n",
            "Episode 706: 11.0\n",
            "Episode 707: 10.0\n",
            "Episode 708: 38.0\n",
            "Episode 709: 15.0\n",
            "Episode 710: 12.0\n",
            "Episode 711: 14.0\n",
            "Episode 712: 9.0\n",
            "Episode 713: 12.0\n",
            "Episode 714: 18.0\n",
            "Episode 715: 12.0\n",
            "Episode 716: 27.0\n",
            "Episode 717: 14.0\n",
            "Episode 718: 15.0\n",
            "Episode 719: 14.0\n",
            "Episode 720: 11.0\n",
            "Episode 721: 25.0\n",
            "Episode 722: 23.0\n",
            "Episode 723: 11.0\n",
            "Episode 724: 26.0\n",
            "Episode 725: 30.0\n",
            "Episode 726: 9.0\n",
            "Episode 727: 9.0\n",
            "Episode 728: 14.0\n",
            "Episode 729: 14.0\n",
            "Episode 730: 10.0\n",
            "Episode 731: 20.0\n",
            "Episode 732: 22.0\n",
            "Episode 733: 22.0\n",
            "Episode 734: 15.0\n",
            "Episode 735: 11.0\n",
            "Episode 736: 22.0\n",
            "Episode 737: 12.0\n",
            "Episode 738: 12.0\n",
            "Episode 739: 11.0\n",
            "Episode 740: 18.0\n",
            "Episode 741: 14.0\n",
            "Episode 742: 12.0\n",
            "Episode 743: 12.0\n",
            "Episode 744: 12.0\n",
            "Episode 745: 11.0\n",
            "Episode 746: 21.0\n",
            "Episode 747: 11.0\n",
            "Episode 748: 11.0\n",
            "Episode 749: 11.0\n",
            "Episode 750: 11.0\n",
            "Episode 751: 11.0\n",
            "Episode 752: 17.0\n",
            "Episode 753: 12.0\n",
            "Episode 754: 9.0\n",
            "Episode 755: 17.0\n",
            "Episode 756: 11.0\n",
            "Episode 757: 10.0\n",
            "Episode 758: 8.0\n",
            "Episode 759: 11.0\n",
            "Episode 760: 11.0\n",
            "Episode 761: 39.0\n",
            "Episode 762: 26.0\n",
            "Episode 763: 13.0\n",
            "Episode 764: 20.0\n",
            "Episode 765: 14.0\n",
            "Episode 766: 13.0\n",
            "Episode 767: 18.0\n",
            "Episode 768: 19.0\n",
            "Episode 769: 23.0\n",
            "Episode 770: 29.0\n",
            "Episode 771: 13.0\n",
            "Episode 772: 18.0\n",
            "Episode 773: 10.0\n",
            "Episode 774: 10.0\n",
            "Episode 775: 15.0\n",
            "Episode 776: 21.0\n",
            "Episode 777: 15.0\n",
            "Episode 778: 12.0\n",
            "Episode 779: 14.0\n",
            "Episode 780: 9.0\n",
            "Episode 781: 15.0\n",
            "Episode 782: 10.0\n",
            "Episode 783: 15.0\n",
            "Episode 784: 10.0\n",
            "Episode 785: 13.0\n",
            "Episode 786: 12.0\n",
            "Episode 787: 23.0\n",
            "Episode 788: 12.0\n",
            "Episode 789: 11.0\n",
            "Episode 790: 20.0\n",
            "Episode 791: 9.0\n",
            "Episode 792: 10.0\n",
            "Episode 793: 17.0\n",
            "Episode 794: 10.0\n",
            "Episode 795: 14.0\n",
            "Episode 796: 15.0\n",
            "Episode 797: 17.0\n",
            "Episode 798: 14.0\n",
            "Episode 799: 10.0\n",
            "Episode 800: 25.0\n",
            "Episode 801: 29.0\n",
            "Episode 802: 23.0\n",
            "Episode 803: 17.0\n",
            "Episode 804: 10.0\n",
            "Episode 805: 14.0\n",
            "Episode 806: 16.0\n",
            "Episode 807: 16.0\n",
            "Episode 808: 15.0\n",
            "Episode 809: 16.0\n",
            "Episode 810: 13.0\n",
            "Episode 811: 9.0\n",
            "Episode 812: 26.0\n",
            "Episode 813: 14.0\n",
            "Episode 814: 9.0\n",
            "Episode 815: 18.0\n",
            "Episode 816: 13.0\n",
            "Episode 817: 23.0\n",
            "Episode 818: 15.0\n",
            "Episode 819: 11.0\n",
            "Episode 820: 18.0\n",
            "Episode 821: 25.0\n",
            "Episode 822: 21.0\n",
            "Episode 823: 23.0\n",
            "Episode 824: 14.0\n",
            "Episode 825: 12.0\n",
            "Episode 826: 15.0\n",
            "Episode 827: 11.0\n",
            "Episode 828: 12.0\n",
            "Episode 829: 17.0\n",
            "Episode 830: 10.0\n",
            "Episode 831: 12.0\n",
            "Episode 832: 29.0\n",
            "Episode 833: 12.0\n",
            "Episode 834: 33.0\n",
            "Episode 835: 13.0\n",
            "Episode 836: 14.0\n",
            "Episode 837: 12.0\n",
            "Episode 838: 13.0\n",
            "Episode 839: 13.0\n",
            "Episode 840: 32.0\n",
            "Episode 841: 9.0\n",
            "Episode 842: 18.0\n",
            "Episode 843: 12.0\n",
            "Episode 844: 12.0\n",
            "Episode 845: 17.0\n",
            "Episode 846: 14.0\n",
            "Episode 847: 10.0\n",
            "Episode 848: 10.0\n",
            "Episode 849: 13.0\n",
            "Episode 850: 11.0\n",
            "Episode 851: 10.0\n",
            "Episode 852: 10.0\n",
            "Episode 853: 13.0\n",
            "Episode 854: 14.0\n",
            "Episode 855: 17.0\n",
            "Episode 856: 11.0\n",
            "Episode 857: 28.0\n",
            "Episode 858: 15.0\n",
            "Episode 859: 16.0\n",
            "Episode 860: 13.0\n",
            "Episode 861: 9.0\n",
            "Episode 862: 13.0\n",
            "Episode 863: 11.0\n",
            "Episode 864: 10.0\n",
            "Episode 865: 63.0\n",
            "Episode 866: 9.0\n",
            "Episode 867: 11.0\n",
            "Episode 868: 16.0\n",
            "Episode 869: 11.0\n",
            "Episode 870: 13.0\n",
            "Episode 871: 10.0\n",
            "Episode 872: 28.0\n",
            "Episode 873: 10.0\n",
            "Episode 874: 17.0\n",
            "Episode 875: 15.0\n",
            "Episode 876: 42.0\n",
            "Episode 877: 12.0\n",
            "Episode 878: 14.0\n",
            "Episode 879: 9.0\n",
            "Episode 880: 13.0\n",
            "Episode 881: 11.0\n",
            "Episode 882: 24.0\n",
            "Episode 883: 13.0\n",
            "Episode 884: 13.0\n",
            "Episode 885: 11.0\n",
            "Episode 886: 27.0\n",
            "Episode 887: 12.0\n",
            "Episode 888: 19.0\n",
            "Episode 889: 13.0\n",
            "Episode 890: 10.0\n",
            "Episode 891: 15.0\n",
            "Episode 892: 15.0\n",
            "Episode 893: 17.0\n",
            "Episode 894: 8.0\n",
            "Episode 895: 39.0\n",
            "Episode 896: 23.0\n",
            "Episode 897: 10.0\n",
            "Episode 898: 25.0\n",
            "Episode 899: 13.0\n",
            "Episode 900: 14.0\n",
            "Episode 901: 25.0\n",
            "Episode 902: 14.0\n",
            "Episode 903: 9.0\n",
            "Episode 904: 12.0\n",
            "Episode 905: 50.0\n",
            "Episode 906: 16.0\n",
            "Episode 907: 30.0\n",
            "Episode 908: 11.0\n",
            "Episode 909: 26.0\n",
            "Episode 910: 10.0\n",
            "Episode 911: 14.0\n",
            "Episode 912: 33.0\n",
            "Episode 913: 14.0\n",
            "Episode 914: 14.0\n",
            "Episode 915: 11.0\n",
            "Episode 916: 9.0\n",
            "Episode 917: 15.0\n",
            "Episode 918: 14.0\n",
            "Episode 919: 17.0\n",
            "Episode 920: 10.0\n",
            "Episode 921: 38.0\n",
            "Episode 922: 14.0\n",
            "Episode 923: 12.0\n",
            "Episode 924: 12.0\n",
            "Episode 925: 37.0\n",
            "Episode 926: 12.0\n",
            "Episode 927: 11.0\n",
            "Episode 928: 22.0\n",
            "Episode 929: 11.0\n",
            "Episode 930: 21.0\n",
            "Episode 931: 17.0\n",
            "Episode 932: 9.0\n",
            "Episode 933: 19.0\n",
            "Episode 934: 12.0\n",
            "Episode 935: 14.0\n",
            "Episode 936: 9.0\n",
            "Episode 937: 36.0\n",
            "Episode 938: 14.0\n",
            "Episode 939: 11.0\n",
            "Episode 940: 14.0\n",
            "Episode 941: 12.0\n",
            "Episode 942: 12.0\n",
            "Episode 943: 11.0\n",
            "Episode 944: 21.0\n",
            "Episode 945: 11.0\n",
            "Episode 946: 12.0\n",
            "Episode 947: 9.0\n",
            "Episode 948: 12.0\n",
            "Episode 949: 14.0\n",
            "Episode 950: 9.0\n",
            "Episode 951: 15.0\n",
            "Episode 952: 13.0\n",
            "Episode 953: 12.0\n",
            "Episode 954: 14.0\n",
            "Episode 955: 17.0\n",
            "Episode 956: 23.0\n",
            "Episode 957: 17.0\n",
            "Episode 958: 10.0\n",
            "Episode 959: 15.0\n",
            "Episode 960: 16.0\n",
            "Episode 961: 17.0\n",
            "Episode 962: 14.0\n",
            "Episode 963: 15.0\n",
            "Episode 964: 11.0\n",
            "Episode 965: 26.0\n",
            "Episode 966: 12.0\n",
            "Episode 967: 12.0\n",
            "Episode 968: 14.0\n",
            "Episode 969: 19.0\n",
            "Episode 970: 41.0\n",
            "Episode 971: 13.0\n",
            "Episode 972: 11.0\n",
            "Episode 973: 13.0\n",
            "Episode 974: 12.0\n",
            "Episode 975: 13.0\n",
            "Episode 976: 17.0\n",
            "Episode 977: 20.0\n",
            "Episode 978: 11.0\n",
            "Episode 979: 21.0\n",
            "Episode 980: 13.0\n",
            "Episode 981: 12.0\n",
            "Episode 982: 11.0\n",
            "Episode 983: 11.0\n",
            "Episode 984: 10.0\n",
            "Episode 985: 13.0\n",
            "Episode 986: 10.0\n",
            "Episode 987: 13.0\n",
            "Episode 988: 12.0\n",
            "Episode 989: 11.0\n",
            "Episode 990: 11.0\n",
            "Episode 991: 15.0\n",
            "Episode 992: 11.0\n",
            "Episode 993: 11.0\n",
            "Episode 994: 10.0\n",
            "Episode 995: 12.0\n",
            "Episode 996: 12.0\n",
            "Episode 997: 13.0\n",
            "Episode 998: 16.0\n",
            "Episode 999: 9.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4d6bAU5cK965",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}